{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import time\n",
    "import requests\n",
    "import pandas as pd\n",
    "import re\n",
    "import json\n",
    "import pickle\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver import ActionChains\n",
    "from selenium.common import exceptions\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from selenium.webdriver.common.by import By\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_proxy_ip():\n",
    "    try:\n",
    "        zhima_ip = r'http://webapi.http.zhimacangku.com/getip?num=1&type=1&pro=&city=0&yys=0&port=11&pack=33844&ts=0&ys=0&cs=0&lb=1&sb=0&pb=4&mr=1&regions='\n",
    "        proxy = requests.get(zhima_ip).text[:-2]\n",
    "        time.sleep(1.5)\n",
    "        prox = {\n",
    "            'https': 'https://{}'.format(proxy),\n",
    "            'http': 'http://{}'.format(proxy)}\n",
    "        res = requests.get('http://2018.ip138.com/ic.asp', proxies=prox, timeout=10)\n",
    "        if res.status_code == 200:\n",
    "            return proxy.split(':')\n",
    "        else:\n",
    "            return get_proxy_ip()\n",
    "    except:\n",
    "        return get_proxy_ip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "proxy_url, port = get_proxy_ip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'proxy_url' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-1-d2ecf18da91a>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mproxy_url\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mport\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'proxy_url' is not defined"
     ]
    }
   ],
   "source": [
    "proxy_url, port"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36'"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_agent = [\n",
    "    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko)Chrome/50.0.2661.102 Safari/537.36',\n",
    "    'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36',\n",
    "    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/46.0.2486.0 Safari/537.36 Edge/13.10586',\n",
    "    'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)',\n",
    "    'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.11 (KHTML, like Gecko) Chrome/20.0.1132.11 TaoBrowser/2.0 Safari/536.11',\n",
    "    \"Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; .NET4.0C; .NET4.0E; .NET CLR 2.0.50727; .NET CLR 3.0.30729; .NET CLR 3.5.30729; InfoPath.3; rv:11.0) like Gecko\",\n",
    "    \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11\",\n",
    "    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/630.0.3538.77 Safari/537.36',\n",
    "]\n",
    "headers = random.choice(user_agent)\n",
    "headers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "user = 'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; .NET4.0C; .NET4.0E; .NET CLR 2.0.50727; .NET CLR 3.0.30729; .NET CLR 3.5.30729; InfoPath.3; rv:11.0) like Gecko'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "fp = webdriver.FirefoxProfile()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "fp.set_preference(\"general.useragent.override\", headers)\n",
    "# fp.set_preference('network.proxy.type', 1)\n",
    "# fp.set_preference('network.proxy.http', proxy_url)\n",
    "# fp.set_preference('network.proxy.http_port', int(port))\n",
    "# fp.set_preference('network.proxy.ssl', proxy_url)\n",
    "# fp.set_preference('network.proxy.ssl_port', int(port))\n",
    "fp.update_preferences()\n",
    "driver = webdriver.Firefox(firefox_profile=fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# driver.get('http://2018.ip138.com/ic.asp')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.set_page_load_timeout(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.firefox_profile.update_preferences()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "driver.firefox_profile.default_preferences['general.useragent.override']\n",
    "# driver.firefox_profile.default_preferences['network.proxy.type'] = 1\n",
    "# driver.firefox_profile.default_preferences['network.proxy.http'] = proxy_url\n",
    "# driver.firefox_profile.default_preferences['network.proxy.http_port'] = int(port)\n",
    "# driver.firefox_profile.default_preferences['network.proxy.ssl'] = proxy_url\n",
    "# driver.firefox_profile.default_preferences['network.proxy.ssl_port'] = port\n",
    "# driver.firefox_profile.update_preferences()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def script_exec():\n",
    "    script_webdriver = '''Object.defineProperties(navigator,{\n",
    "    webdriver:{\n",
    "    get:() => false\n",
    "    }\n",
    "    })'''\n",
    "    script_language = ''' Object.defineProperty(navigator, 'languages', {\n",
    "        get: () => [\"zh-CN\", \"zh\", \"en-US\", \"en\"],\n",
    "      });'''\n",
    "    driver.execute_script(script_webdriver)\n",
    "    driver.execute_script(script_language)\n",
    "script_exec()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.get('https://hm.m.tmall.com')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_items_url = driver.find_element_by_xpath('/html/body/div[1]/div[1]/div[2]/div[1]/section[1]/div/a[2]').get_attribute('href')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.get(all_items_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "script_exec()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "没有切换成功\n"
     ]
    }
   ],
   "source": [
    "if driver.find_elements_by_css_selector('#sufei-dialog-content'):\n",
    "    iframe = driver.find_elements_by_css_selector('#sufei-dialog-content')[0]\n",
    "    driver.switch_to.frame(iframe)\n",
    "    print('切换到悬浮窗口')\n",
    "    if driver.find_elements_by_xpath('/html/body/div[1]/div/div/div/div[2]/form/div[1]/label'): \n",
    "        print('是登录窗口')\n",
    "    else:\n",
    "        print('不是登录窗口')\n",
    "else:\n",
    "    print('没有切换成功')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "if driver.find_elements_by_css_selector('#TPL_username_1'):\n",
    "    driver.find_element_by_css_selector('#TPL_username_1').send_keys('17621780176')\n",
    "    time.sleep(1)\n",
    "    driver.find_element_by_css_selector('#TPL_password_1').send_keys('www.,194928.,com')\n",
    "    time.sleep(random.random()*3)\n",
    "    driver.find_element_by_css_selector('#J_SubmitStatic').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "if driver.find_elements_by_id('J_Message'):\n",
    "    driver.find_element_by_css_selector('#TPL_password_1').send_keys('www.,194928.,com')\n",
    "    time.sleep(random.random()*10)\n",
    "    action = ActionChains(driver)\n",
    "    source=driver.find_element_by_xpath(\"//*[@id='nc_1_n1t']/span\")#需要滑动的元素\n",
    "    action.click_and_hold(source).perform()\n",
    "    time.sleep(0.5)\n",
    "    action.move_by_offset(206,0)#需要滑动的坐标\n",
    "    time.sleep(1.2)\n",
    "    action.release().perform() #释放鼠标\n",
    "#     script_refresh = 'javascript:noCaptcha.reset(1)'\n",
    "#     driver.execute_script(script_refresh)\n",
    "    time.sleep(2)\n",
    "    driver.find_element_by_css_selector('#J_SubmitStatic').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# driver.find_element_by_css_selector('#J_SubmitStatic').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# driver.find_element_by_css_selector('#TPL_password_1').send_keys('www.,194928.,com')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# action = ActionChains(driver)\n",
    "# source=driver.find_element_by_xpath(\"//*[@id='nc_1_n1t']/span\")#需要滑动的元素\n",
    "# action.click_and_hold(source).perform()\n",
    "# time.sleep(0.5)\n",
    "# action.move_by_offset(206,0)#需要滑动的坐标\n",
    "# time.sleep(1.2)\n",
    "# action.release().perform() #释放鼠标"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# script_refresh = 'javascript:noCaptcha.reset(1)'\n",
    "# driver.execute_script(script_refresh)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# driver.find_element_by_css_selector('#J_SubmitStatic').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 缺少处理如何滑动解锁的步骤"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.get('https://only.m.tmall.com/shop/shop_auction_search.htm?suid=356060330&sort=default')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.refresh()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "wrong\n"
     ]
    }
   ],
   "source": [
    "if driver.find_elements_by_css_selector('div.errloading span.nc-lang-cnt'):\n",
    "    again = driver.find_elements_by_css_selector('div.errloading span.nc-lang-cnt')[0].text\n",
    "    script_refresh = 'javascript:noCaptcha.reset(1)'\n",
    "    driver.execute_script(script_refresh)\n",
    "    print(again)\n",
    "else:\n",
    "    print('wrong')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1, 2, 4, 9, 18, 38, 80, 164]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_track():\n",
    "    step = round(random.uniform(0.60, 1),2)\n",
    "    arr = [np.e ** x for x in np.arange(0.01, 5.827, step)]\n",
    "    arr.append(np.e**random.uniform(5.6, 5.83))\n",
    "    need_data = []\n",
    "    for i in range(len(arr))[:-1]:\n",
    "        data = arr[i+1] - arr[i]\n",
    "        if data >= 0:\n",
    "            need_data.append(int(data))\n",
    "    return need_data\n",
    "get_track()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for i in range(10):\n",
    "#     print(sum(get_track()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# iframe2 = driver.find_element_by_css_selector('#sufei-dialog-content')\n",
    "def to_iframe():\n",
    "    try:\n",
    "        iframe = driver.find_element_by_css_selector('#sufei-dialog-content')\n",
    "        driver.switch_to.frame(iframe)\n",
    "        return True\n",
    "    except:\n",
    "        print('no need switch to iframe')\n",
    "        return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "no need switch to iframe\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "to_iframe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def SlideVal():\n",
    "    try:\n",
    "        action = ActionChains(driver)\n",
    "        source = driver.find_element_by_id('nc_1_n1z') #需要滑动的元素\n",
    "        action.click_and_hold(source).perform()\n",
    "        for i in track:\n",
    "            action.move_by_offset(i,0) #需要滑动的坐标\n",
    "        action.release().perform()  #释放鼠标\n",
    "        action.reset_actions()\n",
    "    except exceptions.NoSuchElementException:\n",
    "        pass\n",
    "    return '----滑动解锁完成'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# driver.refresh()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# to_iframe()\n",
    "track = get_track()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'----滑动解锁完成'"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SlideVal()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# if driver.find_elements_by_xpath('/html/body/div[1]/div/div/div/div[2]/form/div[1]/label'):\n",
    "#     print('滑动验证窗口')\n",
    "# else:\n",
    "#     track = get_track()\n",
    "#     to_iframe = outside_code()\n",
    "#     if to_iframe:\n",
    "#         print(SlideVal())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "# driver.switch_to.default_content()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# iframe = driver.find_element_by_id('sufei-dialog-content')\n",
    "# driver.switch_to.frame(iframe)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# action.reset_actions()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# source = driver.find_element_by_xpath('//*[@id=\"nc_1_n1z\"]')  #需要滑动的元素"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # source = driver.find_element_by_id('nc_1_n1z') #需要滑动的元素\n",
    "# source = driver.find_element_by_class_name('nc_iconfont.btn_slide')\n",
    "# action = ActionChains(driver)\n",
    "# action.click_and_hold(source).perform()\n",
    "# action = ActionChains(driver)\n",
    "# for i in get_track():\n",
    "#     action.move_by_offset(i, 0)\n",
    "# action.release().perform()  #释放鼠标"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# source = driver.find_element_by_class_name('nc_iconfont.btn_slide')\n",
    "# action = ActionChains(driver)\n",
    "# action.click_and_hold(source).perform() #按住鼠标\n",
    "# action.move_by_offset(256,0) #需要滑动的坐标\n",
    "# action.release().perform()  #释放鼠标"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# iframe2 = driver.find_element_by_css_selector('#sufei-dialog-content')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.get('https://semir.m.tmall.com/shop/shop_auction_search.htm?sort=default')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# driver.execute_script(\"window.scrollBy(0, 700)\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# driver.execute_script(\"window.scrollTo(0, document.body.scrollHeight)\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for i in range(10):\n",
    "#     a = i * 10000\n",
    "#     b = (i+1) * 10000\n",
    "#     script = \"window.scrollTo({},{})\".format(a, b)\n",
    "#     driver.execute_script(script)\n",
    "#     time.sleep(random.randint(1, 3) + 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# driver.find_elements_by_tag_name('iframe')\n",
    "# sufei-dialog-content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'https://semir.m.tmall.com/shop/shop_auction_search.htm?sort=default'"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "driver.current_url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "def func_totalpage():\n",
    "    firt_page_url = 'https://{}.m.tmall.com/shop/shop_auction_search.do?ajson=1&_tm_source=tmallsearch&sort=s&p=1&page_size=12&from=h5&callback=jsonp_{}'.format(shop_name, num)\n",
    "    driver.get(firt_page_url)\n",
    "    html = driver.page_source\n",
    "    try:\n",
    "        infos = re.findall(r'\\(({.*})\\)', html)[0]\n",
    "        infos = json.loads(infos)\n",
    "        totalpage = int(infos.get('total_page'))\n",
    "        return int(totalpage*1.04)\n",
    "    except (IndexError, TypeError):\n",
    "        raise IndexError"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_totalpage():\n",
    "    locator1 = (By.ID, 'sufei-dialog-content')\n",
    "    times = 1\n",
    "    while True:\n",
    "        try:\n",
    "            totalpage = func_totalpage()\n",
    "            return totalpage\n",
    "        except:\n",
    "            driver.get('https://{}.m.tmall.com/shop/shop_auction_search.htm?sort=default'.format(shop_name))\n",
    "            WebDriverWait(driver, 60).until(EC.presence_of_element_located(locator1))\n",
    "            driver.implicitly_wait(60)\n",
    "            toiframe = to_iframe()\n",
    "            if toiframe:\n",
    "                track = get_track()\n",
    "                print(SlideVal())\n",
    "                time.sleep(5)\n",
    "        print('第%s次获取总页数'%times)\n",
    "        times += 1\n",
    "        if times == 3:\n",
    "            return 0\n",
    "        else:\n",
    "            continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "def try_totalpage():\n",
    "    try:\n",
    "        totalpage = get_totalpage()\n",
    "        return totalpage\n",
    "    except:\n",
    "        return try_totalpage()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# 73932125\n",
    "shop_name = 'eichitoo'\n",
    "num = 75727482\n",
    "i = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----滑动解锁完成\n",
      "第1次获取总页数\n",
      "----滑动解锁完成\n",
      "第2次获取总页数\n",
      "0\n"
     ]
    }
   ],
   "source": [
    "# get_totalpage()\n",
    "# func_totalpage()\n",
    "print(get_totalpage())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_one_page(shop_name, i):\n",
    "    print('    第',i,'页正在加载')\n",
    "    url = 'https://{}.m.tmall.com/shop/shop_auction_search.do?ajson=1&_tm_source=tmallsearch&sort=s&p={}&page_size=12&from=h5&callback=jsonp_{}'.format(shop_name, i, num)\n",
    "    driver.get(url)\n",
    "    time.sleep(random.random() + 1)\n",
    "    html = driver.page_source\n",
    "    try:\n",
    "        infos = re.findall(r'\\(({.*})\\)', html)[0]\n",
    "        infos = json.loads(infos)\n",
    "        products = infos.get('items')\n",
    "        for one_item in products:\n",
    "            one_item['store_name'] = shop_name\n",
    "            one_item['page'] = str(i)\n",
    "        return products\n",
    "    except (IndexError, TypeError):\n",
    "        raise IndexError"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get_one_page(shop_name, i, num)\n",
    "blank = {\n",
    "        \"item_id\":0,\n",
    "        \"title\":\"0\",\n",
    "        \"img\":\"0\",\n",
    "        \"store_name\": \"0\",\n",
    "        \"page\": 0,\n",
    "        \"sold\":\"0\",\n",
    "        \"quantity\":0,\n",
    "        \"totalSoldQuantity\":0,\n",
    "        \"url\":\"0\",\n",
    "        \"price\":\"0\",\n",
    "        \"titleUnderIconList\": \"0\"\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_one_store():\n",
    "    locator1 = (By.ID, 'sufei-dialog-content')\n",
    "    times = 1\n",
    "    while True:\n",
    "        try:\n",
    "            products = get_one_page(shop_name, i) #获取每页的数据\n",
    "            print('    第 {} 页 数据抓取完成\\n'.format(i))\n",
    "            return products\n",
    "        except IndexError:\n",
    "            driver.get('https://{}.m.tmall.com/shop/shop_auction_search.htm?sort=default'.format(shop_name))\n",
    "            WebDriverWait(driver, 60).until(EC.presence_of_element_located(locator1))\n",
    "            driver.implicitly_wait(60)\n",
    "            toiframe = to_iframe()\n",
    "            if toiframe:\n",
    "                track = get_track()\n",
    "                print(SlideVal())\n",
    "                time.sleep(random.random()*5)\n",
    "        times += 1\n",
    "        if times == 5:\n",
    "            print('    第 {} 页 数据抓取失败\\n'.format(i))\n",
    "            return blank\n",
    "        else:\n",
    "            continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def try_one_store(i):\n",
    "#     try:\n",
    "#         one_page = get_one_store(i)\n",
    "#     except:\n",
    "#         return try_one_store(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_one_url():\n",
    "    \"\"\"从文件获取需要爬取的url链接\"\"\"\n",
    "    url_file = r\"D:\\Python\\TmallCrawler\\file\\Tmall_shop_lglat\"\n",
    "    with open(url_file, 'r', encoding='utf-8') as f:\n",
    "        csv_reader = f.read().split('\\n')\n",
    "        for row in csv_reader:\n",
    "            yield row.split('\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "爱居兔 is beginning\n",
      "71\n",
      "卡宾 is beginning\n",
      "76\n",
      "马克华菲 is beginning\n",
      "50\n",
      "森马 is beginning\n",
      "90\n"
     ]
    }
   ],
   "source": [
    "pages = []\n",
    "for x in get_one_url():\n",
    "    brand_name, shop_url = x\n",
    "    shop_name = shop_url[8:-12]\n",
    "    print('{} is beginning'.format(brand_name))\n",
    "    num = random.randint(127482, 75727482)\n",
    "    totalpage = get_totalpage()\n",
    "    print(totalpage)\n",
    "    pages.append([brand_name, shop_name, totalpage])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "爱居兔 have 71 page items\n",
      "当前页面是: 第 41 页\n",
      "    第 41 页正在加载\n",
      "    第 41 页 数据抓取完成\n",
      "\n",
      "当前页面是: 第 42 页\n",
      "    第 42 页正在加载\n",
      "    第 42 页 数据抓取完成\n",
      "\n",
      "当前页面是: 第 43 页\n",
      "    第 43 页正在加载\n",
      "    第 43 页 数据抓取完成\n",
      "\n",
      "当前页面是: 第 44 页\n",
      "    第 44 页正在加载\n",
      "    第 44 页 数据抓取完成\n",
      "\n",
      "当前页面是: 第 45 页\n",
      "    第 45 页正在加载\n",
      "    第 45 页 数据抓取完成\n",
      "\n",
      "当前页面是: 第 46 页\n",
      "    第 46 页正在加载\n",
      "    第 46 页 数据抓取完成\n",
      "\n",
      "当前页面是: 第 47 页\n",
      "    第 47 页正在加载\n",
      "    第 47 页 数据抓取完成\n",
      "\n",
      "当前页面是: 第 48 页\n",
      "    第 48 页正在加载\n",
      "    第 48 页 数据抓取完成\n",
      "\n",
      "当前页面是: 第 49 页\n",
      "    第 49 页正在加载\n",
      "    第 49 页 数据抓取完成\n",
      "\n",
      "当前页面是: 第 50 页\n",
      "    第 50 页正在加载\n",
      "    第 50 页 数据抓取完成\n",
      "\n",
      "当前页面是: 第 51 页\n",
      "    第 51 页正在加载\n",
      "    第 51 页 数据抓取完成\n",
      "\n",
      "当前页面是: 第 52 页\n",
      "    第 52 页正在加载\n",
      "    第 52 页 数据抓取完成\n",
      "\n",
      "当前页面是: 第 53 页\n",
      "    第 53 页正在加载\n",
      "    第 53 页 数据抓取完成\n",
      "\n",
      "当前页面是: 第 54 页\n",
      "    第 54 页正在加载\n",
      "    第 54 页 数据抓取完成\n",
      "\n",
      "当前页面是: 第 55 页\n",
      "    第 55 页正在加载\n",
      "    第 55 页 数据抓取完成\n",
      "\n",
      "当前页面是: 第 56 页\n",
      "    第 56 页正在加载\n",
      "    第 56 页 数据抓取完成\n",
      "\n",
      "当前页面是: 第 57 页\n",
      "    第 57 页正在加载\n",
      "    第 57 页 数据抓取完成\n",
      "\n",
      "当前页面是: 第 58 页\n",
      "    第 58 页正在加载\n",
      "    第 58 页 数据抓取完成\n",
      "\n",
      "当前页面是: 第 59 页\n",
      "    第 59 页正在加载\n",
      "    第 59 页 数据抓取完成\n",
      "\n",
      "当前页面是: 第 60 页\n",
      "    第 60 页正在加载\n",
      "    第 60 页 数据抓取完成\n",
      "\n",
      "当前页面是: 第 61 页\n",
      "    第 61 页正在加载\n",
      "    第 61 页 数据抓取完成\n",
      "\n",
      "当前页面是: 第 62 页\n",
      "    第 62 页正在加载\n",
      "    第 62 页 数据抓取完成\n",
      "\n",
      "当前页面是: 第 63 页\n",
      "    第 63 页正在加载\n",
      "    第 63 页 数据抓取完成\n",
      "\n",
      "当前页面是: 第 64 页\n",
      "    第 64 页正在加载\n",
      "    第 64 页 数据抓取完成\n",
      "\n",
      "当前页面是: 第 65 页\n",
      "    第 65 页正在加载\n",
      "    第 65 页 数据抓取完成\n",
      "\n",
      "当前页面是: 第 66 页\n",
      "    第 66 页正在加载\n",
      "    第 66 页 数据抓取完成\n",
      "\n",
      "当前页面是: 第 67 页\n",
      "    第 67 页正在加载\n",
      "    第 67 页 数据抓取完成\n",
      "\n",
      "当前页面是: 第 68 页\n",
      "    第 68 页正在加载\n",
      "    第 68 页 数据抓取完成\n",
      "\n",
      "当前页面是: 第 69 页\n",
      "    第 69 页正在加载\n",
      "    第 69 页 数据抓取完成\n",
      "\n",
      "当前页面是: 第 70 页\n",
      "    第 70 页正在加载\n",
      "    第 70 页 数据抓取完成\n",
      "\n",
      "当前页面是: 第 71 页\n",
      "    第 71 页正在加载\n",
      "    第 71 页 数据抓取完成\n",
      "\n",
      "爱居兔 is finished!\n",
      "卡宾 have 76 page items\n",
      "当前页面是: 第 41 页\n",
      "    第 41 页正在加载\n",
      "----滑动解锁完成\n",
      "    第 41 页正在加载\n",
      "    第 41 页 数据抓取完成\n",
      "\n",
      "当前页面是: 第 42 页\n",
      "    第 42 页正在加载\n",
      "----滑动解锁完成\n",
      "    第 42 页正在加载\n"
     ]
    }
   ],
   "source": [
    "all_page = []\n",
    "for brand_name, shop_name, totalpage in pages:\n",
    "    print('{} have {} page items'.format(brand_name, totalpage))\n",
    "    for i in range(41, totalpage+1):\n",
    "        print('当前页面是: 第', i, '页')\n",
    "        one_page = get_one_store()\n",
    "        with open('d:\\\\test\\\\{}'.format(brand_name), 'a+', encoding='utf-8', errors='ignore') as f:\n",
    "            f.write(str(all_page))\n",
    "        all_page.extend(one_page)\n",
    "    df = pd.DataFrame(all_page)\n",
    "    df.to_csv('d:\\\\test\\\\{}_tmall.csv'.format(brand_name), sep=',', mode='a+', encoding='gbk')\n",
    "    print('{} is finished!'.format(brand_name))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# shop_name\n",
    "driver.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# df = pd.DataFrame(all_page)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open('test', 'wb') as f:\n",
    "#     f.write(pickle.dumps(all_page))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
